"""抖音热点爬虫模块"""
import asyncio
import hashlib
import json
import logging
import random
import time
import uuid
from datetime import datetime, timedelta
from typing import Dict, Any, List, Optional, Tuple
from urllib.parse import urlencode, quote

import aiohttp
from aiohttp import ClientSession, ClientTimeout

from .base_spider import BaseSpider

logger = logging.getLogger(__name__)


class DouyinHotSpider(BaseSpider):
    """抖音热点爬虫实现"""
    
    # API端点配置
    API_ENDPOINTS = {
        'hot_search': 'https://www.douyin.com/aweme/v1/web/hot/search/list/',  # 热搜榜
        'challenge': 'https://www.douyin.com/aweme/v1/web/challenge/hot/',     # 挑战榜
        'music': 'https://www.douyin.com/aweme/v1/web/music/hot/',            # 音乐榜
        'feed': 'https://www.douyin.com/aweme/v1/web/feed/',                  # 视频流
        'comment': 'https://www.douyin.com/aweme/v1/web/comment/list/',       # 评论列表
        'web_search': 'https://www.douyin.com/aweme/v1/web/search/item/',     # Web搜索接口
    }
    
    # 分类关键词映射
    CATEGORY_KEYWORDS = {
        "搞笑": ["搞笑", "段子", "恶搞", "笑死", "哈哈", "幽默", "整蛊", "沙雕", "逗比"],
        "知识": ["知识", "科普", "教程", "技巧", "干货", "学习", "教育", "讲解", "教学"],
        "美食": ["美食", "吃货", "做饭", "菜谱", "餐厅", "烹饪", "料理", "探店", "测评"],
        "旅行": ["旅行", "旅游", "景点", "打卡", "攻略", "风景", "民宿", "出游", "自驾"],
        "体育": ["体育", "运动", "健身", "篮球", "足球", "跑步", "瑜伽", "减肥", "训练"],
        "游戏": ["游戏", "电竞", "王者", "吃鸡", "原神", "主播", "直播", "解说", "攻略"],
        "音乐": ["音乐", "唱歌", "翻唱", "原创", "乐器", "DJ", "说唱", "舞蹈", "跳舞"],
        "时尚": ["时尚", "穿搭", "美妆", "护肤", "发型", "outfit", "ootd", "化妆", "口红"],
        "宠物": ["宠物", "猫", "狗", "萌宠", "铲屎官", "喵星人", "汪星人", "猫咪", "狗狗"],
        "科技": ["科技", "数码", "手机", "电脑", "AI", "评测", "开箱", "黑科技", "测评"]
    }
    
    def __init__(self):
        """初始化抖音爬虫"""
        super().__init__(name="DouyinHotSpider")
        
        # 设备参数
        self.device_params = self._generate_device_params()
        
        # 用户代理池（移动端为主）
        self.user_agents = [
            'com.ss.android.ugc.aweme/190102 (Linux; U; Android 11; zh_CN; Pixel 5; Build/RQ3A.210905.001; Cronet/TTNetVersion:60889b37 2022-10-19 QuicVersion:12a1d5c5 2022-10-17)',
            'com.ss.android.ugc.aweme/190102 (Linux; U; Android 12; zh_CN; SM-G9910; Build/SP1A.210812.016; Cronet/TTNetVersion:60889b37 2022-10-19 QuicVersion:12a1d5c5 2022-10-17)',
            'com.ss.android.ugc.aweme/190102 (Linux; U; Android 10; zh_CN; MI 9; Build/QKQ1.190825.002; Cronet/TTNetVersion:60889b37 2022-10-19 QuicVersion:12a1d5c5 2022-10-17)',
            'Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Douyin/19.1.0',
            'Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Douyin/19.1.0',
        ]
        
        # 请求头模板
        self.headers_template = {
            'Accept': 'application/json, text/plain, */*',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Cache-Control': 'no-cache',
            'Pragma': 'no-cache',
            'Referer': 'https://www.douyin.com/',
            'Sec-Ch-Ua': '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"',
            'Sec-Ch-Ua-Mobile': '?0',
            'Sec-Ch-Ua-Platform': '"Windows"',
            'Sec-Fetch-Dest': 'empty',
            'Sec-Fetch-Mode': 'cors',
            'Sec-Fetch-Site': 'same-origin',
        }
        
        # 限流配置
        self.rate_limit = 2.0  # 基础间隔2秒
        self.random_delay = (0, 2.0)  # 随机延迟0-2秒
        self.max_concurrent = 3  # 最大并发数
        self.semaphore = asyncio.Semaphore(self.max_concurrent)
        
        # Cookie池（需要定期更新）
        self.cookies = []
        self.current_cookie_index = 0
        
    def _generate_device_params(self) -> Dict[str, str]:
        """生成设备参数
        
        Returns:
            设备参数字典
        """
        device_id = str(random.randint(10000000000000000, 99999999999999999))
        install_id = str(random.randint(10000000000000000, 99999999999999999))
        iid = str(random.randint(10000000000000000, 99999999999999999))
        
        return {
            'device_id': device_id,
            'iid': iid,
            'install_id': install_id,
            'device_platform': 'android',
            'device_type': 'Pixel 5',
            'os_version': '11',
            'app_version': '19.1.0',
            'version_code': '190102',
            'channel': 'googleplay',
            'language': 'zh',
            'region': 'CN',
            'app_name': 'aweme',
            'app_id': '1128',
            'os': 'android',
            'ac': 'wifi',
            'update_version_code': '19102',
            'aid': '1128',
            'resolution': '1080*1920',
            'dpi': '420',
            'cdid': str(uuid.uuid4()),
        }
    
    def _generate_x_bogus(self, params: str) -> str:
        """生成X-Bogus签名（简化版本）
        
        Args:
            params: URL参数字符串
            
        Returns:
            X-Bogus签名
            
        Note:
            实际的X-Bogus算法非常复杂，这里仅作示例
            生产环境建议使用专门的签名服务或逆向完整算法
        """
        # 简化的签名生成（示例）
        timestamp = str(int(time.time()))
        sign_str = f"{params}{timestamp}douyin"
        sign_hash = hashlib.md5(sign_str.encode()).hexdigest()
        
        # 实际的X-Bogus格式更复杂，这里仅示例
        x_bogus = f"DFSzswVLQDc{sign_hash[:16]}ANjTD4b"
        return x_bogus
    
    async def _make_request(self, url: str, params: Dict[str, Any] = None) -> Optional[Dict]:
        """发起请求并处理响应
        
        Args:
            url: 请求URL
            params: 查询参数
            
        Returns:
            响应JSON数据
        """
        async with self.semaphore:
            # 添加设备参数
            if params is None:
                params = {}
            params.update(self.device_params)
            params['ts'] = str(int(time.time()))
            
            # 生成签名
            params_str = urlencode(params)
            x_bogus = self._generate_x_bogus(params_str)
            params['X-Bogus'] = x_bogus
            
            # 设置请求头
            headers = self.headers_template.copy()
            headers['User-Agent'] = random.choice(self.user_agents)
            
            # 添加Cookie（如果有）
            if self.cookies:
                headers['Cookie'] = self.cookies[self.current_cookie_index % len(self.cookies)]
                self.current_cookie_index = (self.current_cookie_index + 1) % len(self.cookies)
            
            # 限流延迟
            delay = self.rate_limit + random.uniform(*self.random_delay)
            await asyncio.sleep(delay)
            
            try:
                async with self.session.get(url, params=params, headers=headers) as response:
                    if response.status == 200:
                        data = await response.json()
                        if data.get('status_code') == 0:
                            return data
                        else:
                            logger.warning(f"API返回错误: {data.get('status_msg', 'Unknown error')}")
                    else:
                        logger.warning(f"HTTP错误 {response.status}: {url}")
                        
            except aiohttp.ClientError as e:
                logger.error(f"请求失败: {e}")
            except json.JSONDecodeError as e:
                logger.error(f"JSON解析失败: {e}")
            except Exception as e:
                logger.error(f"未知错误: {e}")
                
            return None
    
    async def fetch_hot_search(self) -> List[Dict[str, Any]]:
        """获取热搜榜数据
        
        Returns:
            热搜数据列表
        """
        logger.info("开始获取抖音热搜榜")
        
        data = await self._make_request(self.API_ENDPOINTS['hot_search'])
        if not data:
            logger.warning("获取热搜榜失败，尝试Web端接口")
            return await self._fetch_web_hot_search()
        
        hot_list = []
        word_list = data.get('data', {}).get('word_list', [])
        
        for idx, item in enumerate(word_list[:50], 1):  # 获取Top 50
            hot_item = {
                'item_id': self.generate_hash(f"douyin_hot_search_{item.get('word', '')}"),
                'item_type': 'hot_search',
                'title': item.get('word', ''),
                'description': item.get('sentence', ''),
                'source': 'douyin',
                'ranking': {
                    'position': idx,
                    'board_type': 'hot_search',
                    'heat_value': item.get('hot_value', 0),
                },
                'metrics': {
                    'search_count': item.get('search_count', 0),
                    'video_count': item.get('video_count', 0),
                },
                'fetch_time': datetime.now(),
                'update_time': datetime.now(),
                'expire_at': datetime.now() + timedelta(days=30),
            }
            hot_list.append(hot_item)
        
        logger.info(f"成功获取 {len(hot_list)} 条热搜数据")
        return hot_list
    
    async def _fetch_web_hot_search(self) -> List[Dict[str, Any]]:
        """从Web端获取热搜（备用方案）
        
        Returns:
            热搜数据列表
        """
        # Web端备用接口实现
        url = "https://www.douyin.com/hot"
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Referer': 'https://www.douyin.com/',
        }
        
        try:
            async with self.session.get(url, headers=headers) as response:
                if response.status == 200:
                    # 这里需要解析HTML或查找数据接口
                    # 简化处理，返回空列表
                    logger.info("Web端热搜接口需要进一步实现")
                    return []
        except Exception as e:
            logger.error(f"Web端请求失败: {e}")
            
        return []
    
    async def fetch_challenge_hot(self) -> List[Dict[str, Any]]:
        """获取热门挑战榜
        
        Returns:
            挑战榜数据列表
        """
        logger.info("开始获取抖音热门挑战榜")
        
        data = await self._make_request(self.API_ENDPOINTS['challenge'])
        if not data:
            logger.warning("获取挑战榜失败")
            return []
        
        challenge_list = []
        challenges = data.get('data', {}).get('challenge_list', [])
        
        for idx, item in enumerate(challenges[:30], 1):
            challenge_item = {
                'item_id': str(item.get('cid', '')),
                'item_type': 'challenge',
                'title': item.get('cha_name', ''),
                'description': item.get('desc', ''),
                'source': 'douyin',
                'topic_stats': {
                    'participant_count': item.get('user_count', 0),
                    'video_count': item.get('view_count', 0),
                    'total_plays': item.get('play_count', 0),
                },
                'ranking': {
                    'position': idx,
                    'board_type': 'challenge',
                    'heat_value': item.get('hot_value', 0),
                },
                'media': {
                    'cover_url': item.get('cover_url', ''),
                },
                'fetch_time': datetime.now(),
                'update_time': datetime.now(),
                'expire_at': datetime.now() + timedelta(days=30),
            }
            challenge_list.append(challenge_item)
        
        logger.info(f"成功获取 {len(challenge_list)} 条挑战数据")
        return challenge_list
    
    async def fetch_music_hot(self) -> List[Dict[str, Any]]:
        """获取热门音乐榜
        
        Returns:
            音乐榜数据列表
        """
        logger.info("开始获取抖音热门音乐榜")
        
        data = await self._make_request(self.API_ENDPOINTS['music'])
        if not data:
            logger.warning("获取音乐榜失败")
            return []
        
        music_list = []
        musics = data.get('data', {}).get('music_list', [])
        
        for idx, item in enumerate(musics[:30], 1):
            music_info = item.get('music_info', {})
            music_item = {
                'item_id': str(music_info.get('mid', '')),
                'item_type': 'music',
                'title': music_info.get('title', ''),
                'description': music_info.get('author', ''),
                'source': 'douyin',
                'media': {
                    'music': {
                        'name': music_info.get('title', ''),
                        'author': music_info.get('author', ''),
                        'usage_count': item.get('use_count', 0),
                        'duration': music_info.get('duration', 0),
                        'cover_url': music_info.get('cover_url', ''),
                    }
                },
                'ranking': {
                    'position': idx,
                    'board_type': 'music',
                    'heat_value': item.get('hot_value', 0),
                },
                'metrics': {
                    'usage_count': item.get('use_count', 0),
                },
                'fetch_time': datetime.now(),
                'update_time': datetime.now(),
                'expire_at': datetime.now() + timedelta(days=30),
            }
            music_list.append(music_item)
        
        logger.info(f"成功获取 {len(music_list)} 条音乐数据")
        return music_list
    
    async def fetch_hot_videos(self, count: int = 50) -> List[Dict[str, Any]]:
        """获取热门视频信息
        
        Args:
            count: 获取数量
            
        Returns:
            视频数据列表
        """
        logger.info(f"开始获取抖音热门视频 (目标: {count})")
        
        video_list = []
        cursor = 0
        
        while len(video_list) < count:
            params = {
                'type': '0',  # 0: 推荐, 1: 同城
                'max_cursor': str(cursor),
                'count': '20',
            }
            
            data = await self._make_request(self.API_ENDPOINTS['feed'], params)
            if not data:
                logger.warning("获取视频流失败")
                break
            
            aweme_list = data.get('aweme_list', [])
            if not aweme_list:
                break
            
            for aweme in aweme_list:
                video_item = await self._parse_video_item(aweme)
                if video_item:
                    video_list.append(video_item)
                    
                if len(video_list) >= count:
                    break
            
            # 更新游标
            cursor = data.get('max_cursor', cursor + 20)
            has_more = data.get('has_more', False)
            
            if not has_more:
                break
            
            # 延迟以避免触发限流
            await asyncio.sleep(random.uniform(1, 3))
        
        logger.info(f"成功获取 {len(video_list)} 条视频数据")
        return video_list[:count]
    
    async def _parse_video_item(self, aweme: Dict) -> Optional[Dict[str, Any]]:
        """解析视频数据
        
        Args:
            aweme: 原始视频数据
            
        Returns:
            格式化的视频数据
        """
        try:
            # 提取基础信息
            video_id = aweme.get('aweme_id', '')
            if not video_id:
                return None
            
            # 提取描述和标题
            desc = aweme.get('desc', '')
            title = desc[:100] if desc else '无标题'
            
            # 提取作者信息
            author = aweme.get('author', {})
            author_id = str(author.get('uid', ''))
            nickname = author.get('nickname', '抖音用户')
            
            # 提取统计数据
            statistics = aweme.get('statistics', {})
            
            # 提取音乐信息
            music = aweme.get('music', {})
            
            # 提取话题标签
            hashtags = []
            text_extra = aweme.get('text_extra', [])
            for extra in text_extra:
                if extra.get('type') == 1:  # 话题类型
                    hashtags.append(extra.get('hashtag_name', ''))
            
            # 自动分类
            categories = self._auto_categorize(f"{title} {' '.join(hashtags)}")
            
            # 构建视频数据
            video_item = {
                'item_id': video_id,
                'item_type': 'video',
                'title': title,
                'description': desc[:500] if desc else '',
                'source': 'douyin',
                'category': '|'.join(categories[:3]),  # 最多3个分类
                'author': {
                    'user_id_hash': self.generate_hash(author_id),
                    'nickname_masked': self._mask_nickname(nickname),
                    'follower_level': self._get_follower_level(author.get('follower_count', 0)),
                    'verification': self._get_verification_type(author.get('custom_verify', '')),
                },
                'metrics': {
                    'likes': statistics.get('digg_count', 0),
                    'comments': statistics.get('comment_count', 0),
                    'shares': statistics.get('share_count', 0),
                    'plays': statistics.get('play_count', 0),
                    'collects': statistics.get('collect_count', 0),
                },
                'media': {
                    'cover_url': aweme.get('video', {}).get('cover', {}).get('url_list', [''])[0],
                    'duration': aweme.get('video', {}).get('duration', 0) // 1000,  # 转换为秒
                    'music': {
                        'name': music.get('title', ''),
                        'author': music.get('author', ''),
                        'usage_count': music.get('user_count', 0),
                    } if music else None,
                },
                'hashtags': hashtags,
                'fetch_time': datetime.now(),
                'update_time': datetime.now(),
                'expire_at': datetime.now() + timedelta(days=30),
            }
            
            return video_item
            
        except Exception as e:
            logger.error(f"解析视频数据失败: {e}")
            return None
    
    def _mask_nickname(self, nickname: str) -> str:
        """昵称脱敏处理
        
        Args:
            nickname: 原始昵称
            
        Returns:
            脱敏后的昵称
        """
        if not nickname or len(nickname) <= 2:
            return "抖**友"
        
        if len(nickname) <= 4:
            return f"{nickname[0]}**{nickname[-1]}"
        
        return f"{nickname[:2]}**{nickname[-2:]}"
    
    def _get_follower_level(self, count: int) -> str:
        """获取粉丝数级别
        
        Args:
            count: 粉丝数
            
        Returns:
            粉丝数级别描述
        """
        if count >= 10000000:
            return "1000万+"
        elif count >= 1000000:
            return "100万+"
        elif count >= 100000:
            return "10万+"
        elif count >= 10000:
            return "1万+"
        elif count >= 1000:
            return "1000+"
        else:
            return str(count)
    
    def _get_verification_type(self, verify_str: str) -> str:
        """获取认证类型
        
        Args:
            verify_str: 认证字符串
            
        Returns:
            认证类型
        """
        if not verify_str:
            return ""
        
        if "企业" in verify_str or "官方" in verify_str:
            return "blue_v"
        elif "认证" in verify_str:
            return "yellow_v"
        else:
            return "verified"
    
    def _auto_categorize(self, text: str) -> List[str]:
        """自动分类
        
        Args:
            text: 待分类文本
            
        Returns:
            分类列表
        """
        categories = []
        scores = {}
        
        # 计算每个分类的匹配分数
        for category, keywords in self.CATEGORY_KEYWORDS.items():
            score = 0
            for keyword in keywords:
                if keyword.lower() in text.lower():
                    score += 1
            if score > 0:
                scores[category] = score
        
        # 按分数排序并返回前3个
        sorted_categories = sorted(scores.items(), key=lambda x: x[1], reverse=True)
        categories = [cat for cat, _ in sorted_categories[:3]]
        
        # 如果没有匹配到任何分类，返回"其他"
        if not categories:
            categories = ["其他"]
        
        return categories
    
    async def parse(self, html: str, url: str = None) -> List[Dict[str, Any]]:
        """解析页面内容（实现基类抽象方法）
        
        Args:
            html: HTML内容
            url: 页面URL
            
        Returns:
            解析后的数据列表
        """
        # 抖音主要使用API，此方法作为备用
        return []
    
    async def crawl_all(self) -> Dict[str, List[Dict[str, Any]]]:
        """执行完整爬取流程
        
        Returns:
            包含所有类型数据的字典
        """
        logger.info("开始执行抖音热点爬取")
        
        results = {
            'hot_search': [],
            'challenges': [],
            'music': [],
            'videos': [],
        }
        
        # 并发执行各项爬取任务
        tasks = [
            self.fetch_hot_search(),
            self.fetch_challenge_hot(),
            self.fetch_music_hot(),
            self.fetch_hot_videos(30),  # 获取30个热门视频
        ]
        
        try:
            hot_search, challenges, music, videos = await asyncio.gather(*tasks)
            
            results['hot_search'] = hot_search
            results['challenges'] = challenges
            results['music'] = music
            results['videos'] = videos
            
            total_items = sum(len(items) for items in results.values())
            logger.info(f"爬取完成，共获取 {total_items} 条数据")
            
        except Exception as e:
            logger.error(f"爬取过程出错: {e}")
        
        return results
    
    async def update_with_history(self, current_data: Dict, history_data: Dict, threshold: float = 0.1) -> Dict:
        """增量更新数据
        
        Args:
            current_data: 当前数据
            history_data: 历史数据
            threshold: 更新阈值（默认10%）
            
        Returns:
            更新后的数据
        """
        # 检查是否需要更新
        need_update = False
        
        # 比较关键指标
        if current_data.get('item_type') == 'video':
            current_metrics = current_data.get('metrics', {})
            history_metrics = history_data.get('metrics', {})
            
            for key in ['likes', 'comments', 'shares', 'plays']:
                current_val = current_metrics.get(key, 0)
                history_val = history_metrics.get(key, 0)
                
                if history_val > 0:
                    change_rate = abs(current_val - history_val) / history_val
                    if change_rate > threshold:
                        need_update = True
                        break
        
        if need_update:
            # 保留历史数据
            if 'history' not in current_data:
                current_data['history'] = []
            
            # 添加历史记录（最多保留7天）
            history_entry = {
                'timestamp': history_data.get('update_time'),
                'metrics': history_data.get('metrics', {}),
                'ranking': history_data.get('ranking', {}),
            }
            
            current_data['history'].append(history_entry)
            
            # 只保留最近7天的历史
            week_ago = datetime.now() - timedelta(days=7)
            current_data['history'] = [
                h for h in current_data['history']
                if h.get('timestamp', datetime.min) > week_ago
            ]
            
            # 更新时间戳
            current_data['update_time'] = datetime.now()
            
            logger.info(f"数据已更新: {current_data.get('item_id')}")
        else:
            logger.debug(f"数据无需更新: {current_data.get('item_id')}")
            return history_data
        
        return current_data
    
    async def crawl_hot(self) -> List[Dict[str, Any]]:
        """统一接口：获取热搜数据"""
        result = await self.crawl_all()
        # 返回热搜榜数据
        return result.get('hot_search', [])


async def main():
    """测试函数"""
    spider = DouyinHotSpider()
    
    try:
        await spider.start()
        
        # 测试各个功能
        print("\n=== 测试热搜榜 ===")
        hot_search = await spider.fetch_hot_search()
        if hot_search:
            print(f"获取到 {len(hot_search)} 条热搜")
            if hot_search:
                print(f"热搜示例: {hot_search[0].get('title')}")
        
        print("\n=== 测试挑战榜 ===")
        challenges = await spider.fetch_challenge_hot()
        if challenges:
            print(f"获取到 {len(challenges)} 个挑战")
            if challenges:
                print(f"挑战示例: {challenges[0].get('title')}")
        
        print("\n=== 测试音乐榜 ===")
        music = await spider.fetch_music_hot()
        if music:
            print(f"获取到 {len(music)} 首音乐")
            if music:
                print(f"音乐示例: {music[0].get('title')}")
        
        print("\n=== 测试热门视频 ===")
        videos = await spider.fetch_hot_videos(5)
        if videos:
            print(f"获取到 {len(videos)} 个视频")
            for video in videos[:3]:
                print(f"- {video.get('title')[:30]}... (点赞: {video.get('metrics', {}).get('likes', 0)})")
        
    finally:
        await spider.close()


if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    asyncio.run(main())